# -*- coding: utf-8 -*-
'''
Created on 26-11-2012

@author: LONG HOANG GIANG
'''
import os
import sys
import datetime
import workerpool
import re
from CrawlerLib import commonlib
from Model.storymodelv2 import StoryModel
from urlparse import urljoin

def getDetail(url):
    
    tree = commonlib.loadweb(url).build_tree()
    contentNode = tree.xpath("//div[@class='entry-content clearfix']")
    if len(contentNode) == None: raise Exception, 'Xpath Error'
    html = commonlib.html2text(contentNode, allow_image=False).strip()
    html = re.sub(ur"\n", "<br />", html)
    return html

def normalizeTitle(title):
    if 'Giới thiệu' in title: return "Giới thiệu"
    if 'convert' in title or 'Đôi lời gửi các độc giả' in title or 'Bình luận-Bảy ngày ân ái' in title: return ''
    title = re.sub(r"\[(.+)\].+?\[(.+)\]", lambda m: "CHƯƠNG {0}: {1}".format(int(float(re.sub(r"- *P\d+", '', m.group(1).strip()))), m.group(2)), title)
    return title

def getListChapter():
    tree = commonlib.loadweb(URL).build_tree()
    data = []
    for alink in tree.xpath("//ul[@class='lcp_catlist']/li/a"):
        title = normalizeTitle(commonlib.stringify(alink).strip())
        link = alink.get('href', '')
        if link == '' or title == '': continue
        link = urljoin(URL, link)
        data.append({'title': title, 'url': link})
    data.reverse()      # Đảo ngược mảng, do bọn này post chapter mới nhất trên cùng
    
#    def loadPage(item):
#        commonlib.loadweb(item['url'])
#        
#    pool = workerpool.WorkerPool(size=5)
#    pool.map(loadPage, data)
#    pool.shutdown()
#    pool.wait()
    
    return data

def process():
    
    data = getListChapter()
    db = StoryModel(DATABASE)
    db.open(True)
    for chapter in data:
        detail = getDetail(chapter['url'])
        detail = "<b>{0}</b><br /><hr /><br />{1}".format(chapter['title'], detail)
        db.add_story(commonlib.toUpper(chapter['title']), detail, 0)
        print '_____________________________________________'
        print commonlib.toUpper(chapter['title'])
        print '_____________________________________________'
        print detail, "\n\n"
    db.close()
        
    

if __name__ == '__main__':
    
    URL = 'http://vficland.com/truyen-phuong-dong/bay-ngay-an-ai/'
    DATABASE = "/longhoanggiang/database/bayngayanai"
    
    process()
    
    print '> Finished at {0}'.format(datetime.datetime.now())
    os._exit(1)
